import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.*;
import org.jsoup.select.*;

public class Crawler {
	private final static Integer mRetries = 10; // number of retries for failed
												// connection attempts
	private Integer mCrawlLimit = -1;
	private Integer mDownloadMin = -1;

	@SuppressWarnings("serial")
	private final static List<String> mCategories = new ArrayList<String>() {
		{
			/*// First Level
			add("audio-video");
			add("business-enterprise");
			add("communications");
			add("development");
			add("home-education");
			add("games");
			add("graphics");
			add("science-engineering");
			add("security-utilities");
			add("system-administration");*/
			
			// Second Level
			add("audio-video/sound");
			add("audio-video/video");

			add("business-enterprise/scheduling");
			add("business-enterprise/suites");
			add("business-enterprise/ecommerce");
			add("business-enterprise/dtp");
			add("business-enterprise/reportgen");
			add("business-enterprise/knowledgemanagement");
			add("business-enterprise/enterprise");
			add("business-enterprise/financial");
			add("business-enterprise/todo-lists");
			add("business-enterprise/businessmodelling");
			add("business-enterprise/project-management");
			add("business-enterprise/time-tracking");
			add("business-enterprise/insurance");

			add("communications/chat");
			add("communications/rssreaders");
			add("communications/bbs");
			add("communications/conferencing");
			add("communications/email");
			add("communications/fax");
			add("communications/fido");
			add("communications/hamradio");
			add("communications/usenet");
			add("communications/internetphone");
			add("communications/synchronization");
			add("communications/streaming-comms");
			add("communications/telephony");
			add("communications/filesharing");

			add("development/development");
			add("development/database");
			add("development/editors");
			add("development/data-formats");

			add("home-education/religion");
			add("home-education/education");
			add("home-education/printing");
			add("home-education/social-sciences");

			add("games/hobbies");
			add("games/sidescrolling");
			add("games/flightsim");
			add("games/sports");
			add("games/mmorpg");
			add("games/puzzles");
			add("games/realtimetactical");
			add("games/realtimestrategy");
			add("games/firstpersonshooters");
			add("games/turnbasedstrategy");
			add("games/rpg");
			add("games/card-games");
			add("games/mud");
			add("games/console-games");
			add("games/gameframeworks");
			add("games/multiplayer");
			add("games/simulation");
			add("games/boardgames");

			add("graphics/graphics/graphics-capture");
			add("graphics/graphics/graphics-conversion");
			add("graphics/graphics/graphics-editors");
			add("graphics/graphics/3dmodeling");
			add("graphics/graphics/3drendering");
			add("graphics/graphics/fractals");
			add("graphics/graphics/viewers");
			add("graphics/graphics/imagegalleries");
			add("graphics/graphics/presentation");
			add("graphics/graphics/handrec");
			add("graphics/graphics/animation");

			add("science-engineering/chemistry");
			add("science-engineering/informationanalysis");
			add("science-engineering/interfaceengine");
			add("science-engineering/physics");
			add("science-engineering/ai");
			add("science-engineering/astronomy");
			add("science-engineering/visualization");
			add("science-engineering/mapping");
			add("science-engineering/medical");
			add("science-engineering/mechcivileng");
			add("science-engineering/hmi");
			add("science-engineering/medicalphysics");
			add("science-engineering/molecularmech");
			add("science-engineering/quantumcomp");
			add("science-engineering/earth-science");
			add("science-engineering/ecosystem-sciences");
			add("science-engineering/testmeasure");
			add("science-engineering/molecular-science");
			add("science-engineering/buildingauto");
			add("science-engineering/simulations");
			add("science-engineering/robotics");
			add("science-engineering/scada");
			add("science-engineering/mathematics");
			add("science-engineering/linguistics");
			add("science-engineering/eda");
			add("science-engineering/bioinformatics");

			add("security-utilities/storage/archiving");
			add("security-utilities/storage/file-management");
			add("security-utilities/power");
			add("security-utilities/terminals");
			add("security-utilities/security");
			add("security-utilities/logging/log-rotation");
			add("security-utilities/ftp");
			add("security-utilities/graphics/graphics-capture");
			add("security-utilities/loganalysis");

			add("system-administration/softwaredist");
			add("system-administration/benchmark");
			add("system-administration/filesystems");
			add("system-administration/embedded");
			add("system-administration/kernels");
			add("system-administration/cron");
			add("system-administration/setup");
			add("system-administration/logging");
			add("system-administration/networking");
			add("system-administration/power");
			add("system-administration/homeauto");
			add("system-administration/osdistro");
			add("system-administration/shells");
			add("system-administration/distributed-computing");
			add("system-administration/emulators");
			add("system-administration/hardware");
			add("system-administration/system-search");
			add("system-administration/sysadministration");
			add("system-administration/storage");
		}
	};

	private final static String mBaseURL = "http://www.sourceforge.net";
	private final static Integer mPageRows = 100;

	private DataStorage mDataStorageInstance;
	private String mCategory, mDate;
	
	private String mDoneCategory;
	private Integer mDonePage;
	private Integer mCategoryCount = 0;
	
	Crawler(DataStorage ds) {
		this.mDataStorageInstance = ds;

		Date d = new Date();
		Calendar calendar = Calendar.getInstance();
		calendar.setTime(d);
		this.mDate = calendar.get(Calendar.YEAR) + "-"
				+ (calendar.get(Calendar.MONTH) + 1) + "-"
				+ calendar.get(Calendar.DAY_OF_MONTH);
	}

	/*public static void main(String args[]) {
		try {
			JarUtil ju = new JarUtil(JarUtil.class);
			String confFilePath = ju.getJarPath();
			DataStorage ds = new DataStorage();
			ds.init(confFilePath + "/config.ini");
			if (ds.isConnected()) {
				Crawler c = new Crawler(ds);
				c.init(confFilePath + "/config.ini");
				c.run();
			} else {
				System.out.println("MySQL Connection Error!");
			}
		} catch (Exception e) {

		}

	}*/

	public void init(String paramFile) throws FileNotFoundException,
			IOException {
		// Use Properties to load attributes
		Properties props = new Properties();
		try {
			props.load(new FileInputStream(paramFile));
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}
		mCrawlLimit = Integer.valueOf(props.getProperty("crawlLimit"));
		mDownloadMin = Integer.valueOf(props.getProperty("downloadMin"));
	}

	public void run() {
		getProgress();
		startSearch();
	}
	
	void getProgress() {
		mDonePage = -1;
		try {
			FileReader fr = new FileReader("progress.ini");
			BufferedReader br = new BufferedReader(fr);

			String savedDate = br.readLine();
			mDoneCategory = br.readLine();
			mDonePage = Integer.parseInt(br.readLine());

			if (!savedDate.equals(mDate)) {
				mDonePage = -1;
			}
		} catch (IOException e) {
		}
	}
	
	void startSearch() {
		if (mDonePage == -1) {
			for (String category : mCategories) {
				System.out.println("Processing Category: " + category);

				Integer page = 1;
				mCategoryCount = 0;
				while (processCategory(category, page)) {
					++page;
					updateProgress(category, page);
				}
			}
		} else {
			// resume
			boolean found = false;
			for (String category : mCategories) {
				if (!found && category.equals(mDoneCategory) == false)
					continue;
				Integer page = 1;
				if (!found) {
					page = mDonePage;
					found = true;
				}
				System.out.println("Processing Category: " + category);
				mCategoryCount = 0;
				while (processCategory(category, page)) {
					++page;
					updateProgress(category, page);
				}
			}
		}
	}
	
	private boolean processCategory(String category, Integer page) {
		Integer pcnt = 1, tries = 0;
		Elements projects = null;
		Document doc = null;
		Boolean done = false;
		this.mCategory = category;

		while (!done && tries++ <= mRetries * 3) {
			System.out.print("  processing page " + page + "... ");
			try {
				doc = Jsoup.connect(getSearchURL(category, page))
						.timeout(15000).get();
				projects = doc.select("ul.projects li");
				done = true;
			} catch (IOException e) {
				System.out.println("Error! Retrying..");
				processCategory(category, page);
			}
		}

		if (doc != null) {
			// Get staff picks information			
			if (page == 1) {
				Elements li = doc.select(".sidebar-widget ul li");
				if (li.size() != 0) {
					System.out.println("  Inserting staff picks for "
							+ category);
				}
				for (int i = 0; i < li.size(); ++i) {
					Element t = li.get(i);
					String link = t.select("a").attr("href");
					// Sample link string "/projects/chakra/?source=staffpicks"
					// The second element of link.split("/") is the project name
					String name = link.split("/")[2];					
					
					/*String name = t.text();*/

					try {
						mDataStorageInstance.insertStaffPick(category, mDate,
								i + 1, name, link);
					} catch (Exception e) {
						System.out
								.println("  SQL Error when inserting staff picks for "
										+ category);
					}
				}
			}
			System.out.println("Done!");
			for (Element project : projects) {
				if (project.attr("itemtype") != null) {
					String link = project.select("a").attr("href");
					if (link.indexOf("/", 10) < 10)
						continue;
					String pname = link.substring(10, link.indexOf("/", 10));
					Integer ranking = (page - 1) * Crawler.mPageRows + pcnt;
					String name = project.select("span[itemprop=name]").text();
					String description = project.select(
							"p[itemprop=description]").text();
					Integer downloads = Integer.parseInt(project
							.select("strong[itemprop=interactionCount]").text()
							.replace(",", ""));

					if (downloads < mDownloadMin) {
						return false;
					}

					getProjectExtraDetails(pname, ranking, name, description,
							downloads);
					
					pcnt++;
					mCategoryCount++;

					if (mCrawlLimit > -1 && mCategoryCount >= mCrawlLimit) {
						return false;
					}
				}
			}

			
		}

		return (pcnt > 1);
	}
	
	void updateProgress(String category, Integer page) {
		try {
			FileWriter fw = new FileWriter("progress");
			fw.write(mDate + "\n" + category + "\n" + page + "\n");
			fw.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	private static String getSearchURL(String category, Integer page) {
		/*
		 * return Crawler.BaseURL + "/directory/" + category + "/?rows=" +
		 * Crawler.PageRows.toString() + "&page=" + page.toString();
		 */
		// return Crawler.BaseURL + "/directory/" + category +
		// "/os%3Awindows/freshness%3Arecently-updated/" + "?page=" +
		// page.toString();
		String searchURL = new String(Crawler.mBaseURL + "/directory/" + category
				+ "/os:windows/freshness:recently-updated" + "/?rows="
				+ Crawler.mPageRows.toString() + "&page=" + page.toString());
		//System.out.println(searchURL);
		return searchURL;
	}

	private static String getProjectPageURL(String pname) {
		String projectPageURL = new String(Crawler.mBaseURL + "/projects/" + pname + "/");
		//System.out.println(projectPageURL);
		return projectPageURL;
	}

	private static String getProjectExtraDetailsURL(String pname) {
		String projectExtraDetailsURL = new String(Crawler.mBaseURL + "/projects/" + pname + "/more");
		//System.out.println(projectExtraDetailsURL);
		return projectExtraDetailsURL;
	}

	private void getProjectExtraDetails(String pname, Integer ranking,
			String name, String description, Integer downloads) {
		Document doc = null;
		boolean done = false;
		Integer tries = 0;

		while (!done && tries++ <= mRetries) {
			System.out.println("    Getting project extra for " + pname + "... ");
			try {
				doc = Jsoup.connect(getProjectExtraDetailsURL(pname)).get();
				done = true;
			} catch (IOException e) {
				System.out.println("Error! Retrying..");
			}
		}

		if (doc != null) {

			System.out.println("Done!");

			Elements s = doc.select(".screenshots img");
			String screenshot = s.size() > 0 ? s.first().attr("src") : "";
			String detailedDescription = doc.select(
					".details .description p:eq(0)").text();
			String updated = doc.select(".details .description .updated")
					.text().replace("Last Updated: ", "");
			String recommended = doc.select(".details .rating em").text();

			DateParser dp = new DateParser();
			if (dp.parse(updated)) {
				updated = dp.getDateString();
			}

			try {
				mDataStorageInstance.insertProjectDetails(pname, name, mDate, mCategory, ranking,
						downloads, description, detailedDescription, updated,
						screenshot, recommended);
				System.out
						.println("    Inserting project information into Ranking table\n");
			} catch (Exception e) {
				e.printStackTrace();
				System.out
						.println("    Error inserting into MySQL database!\n");
			}
		}
		getProjectPage(pname);
	}

	private void getProjectPage(String pname) {
		// get more detailed information
		Document doc = null;
		Boolean done = false;
		Integer tries = 0;

		while (!done && tries++ <= mRetries) {
			System.out.print("    getting project details for " + pname
					+ "... ");
			try {
				doc = Jsoup.connect(getProjectPageURL(pname)).timeout(50000)
						.get();
				done = true;
			} catch (IOException e) {
				System.out.println("Error! Retrying..");
				e.printStackTrace();
			}
		}
		if (doc != null) {
			System.out.println("Done!");
			
			// User feedback information
			Integer likes, dislikes;
			try {
				likes = Integer
						.parseInt(doc.select(".thumbs-up").text().trim());
			} catch (Exception e) {
				likes = 0;
			}
			try {
				dislikes = Integer.parseInt(doc.select(".thumbs-down").text()
						.trim());
			} catch (Exception e) {
				dislikes = 0;
			}

			try {
				String updated = doc.select(".dateUpdated").attr("datetime");
				
				// Supported OS Information
				Elements projectInfo = doc.select(".project-info");
				Integer has_window = projectInfo.select(".platform-icon.windows").size();
				Integer has_linux = projectInfo.select(".platform-icon.linux").size();
				Integer has_bsd = projectInfo.select(".platform-icon.bsd").size();
				Integer has_mac = projectInfo.select(".platform-icon.mac").size();
				// Category Information
				Elements projectCategoryandLicense = doc.select("#project-categories-and-license");
				Elements projectContainer = projectCategoryandLicense.select(".project-container section");
				Elements subcategories = projectCategoryandLicense.select("a");
				String categories = new String("");
				String license = new String("");
				
				for(Element proInfo: projectContainer){
					if(!proInfo.select("h3:contains(Categories)").isEmpty()){
						Elements eSet = proInfo.select("a");
						//String natlanguageStr = new String("");
						for(Element e: eSet){
							categories += e.text() + ",";
						}
						categories = categories.substring(0, categories.length()-1);
						//System.out.println(categories);
					}
					if(!proInfo.select("section h3:contains(License)").isEmpty()){						
						Elements eSet = proInfo.select("a");
						//String natlanguageStr = new String("");
						for(Element e: eSet){
							license += e.text() + ",";
						}
						license = license.substring(0, license.length()-1);
						//System.out.println(license);
					}
				}
				/*//Li Zhuolun's code on 2012-08-21
				Iterator <Element> it = subcategories.iterator();
				while(it.hasNext()){
					Element e = it.next();
					if(e.attr("itemprop").length() != 0){
						categories += e.text() + ","; 
					}
					
				}
				categories = categories.substring(0, categories.length()-1);	*/			
								
				
				String natlanguageStr = new String("");
				String audienceStr = new String("");
				String uiStr = new String("");
				String programLanguageStr = new String("");
				String awardStr = new String("");
				String registeredDateStr = new String("");
				
				
				Element addtionalInformation = doc.select("#project-additional-trove").first();
				Elements productInfos = addtionalInformation.select(".project-info");
				for(Element proInfo: productInfos){
					if(!proInfo.select("h3:contains(Languages)").isEmpty()){
						Elements eSet = proInfo.select("a");
						//String natlanguageStr = new String("");
						for(Element e: eSet){
							natlanguageStr += e.text() + ",";
						}
						natlanguageStr = natlanguageStr.substring(0, natlanguageStr.length()-1);
						//System.out.println(natlanguageStr);
					}
					if(!proInfo.select("h3:contains(Intended Audience)").isEmpty()){
						Elements eSet = proInfo.select("a");
						//String audienceStr = new String("");
						for(Element e: eSet){
							audienceStr += e.text() + ",";
						}
						audienceStr = audienceStr.substring(0, audienceStr.length()-1);
						//System.out.println(audienceStr);
					}
					if(!proInfo.select("h3:contains(User Interface)").isEmpty()){
						Elements eSet = proInfo.select("a");
						//String uiStr = new String("");
						for(Element e: eSet){
							uiStr += e.text() + ",";
						}
						uiStr = uiStr.substring(0, uiStr.length()-1);
						//System.out.println(uiStr);
					}
					if(!proInfo.select("h3:contains(Programming Language)").isEmpty()){
						Elements eSet = proInfo.select("a");
						//String programLanguageStr = new String("");
						for(Element e: eSet){
							programLanguageStr += e.text() + ",";
						}
						programLanguageStr = programLanguageStr.substring(0, programLanguageStr.length()-1);
						//System.out.println(programLanguageStr);
					}
					if(!proInfo.select("h3:contains(Awards)").isEmpty()){
						Elements eSet = proInfo.select(".content ul li a");
						//String awardStr = new String("");
						for(Element e: eSet){
							awardStr += e.attr("title") + ",";
						}
						awardStr = awardStr.substring(0, awardStr.length()-1);
						System.out.println(awardStr);
					}
					if(!proInfo.select("h3:contains(Registered)").isEmpty()){
					Elements eSet = proInfo.select(".content");
					//String registeredDateStr = new String("");
					for(Element e: eSet){
						registeredDateStr += e.text() + ",";
					}
					registeredDateStr = registeredDateStr.substring(0, registeredDateStr.length()-1);
					//System.out.println(registeredDateStr);
					}					
				}
				
				
				
				// Original Version
				/*String updated = ad.select("span[itemprop=dateUpdated]").text();
				Integer has_window = ad.select(".platform-icon.windows").size();
				Integer has_linux = ad.select(".platform-icon.linux").size();
				Integer has_bsd = ad.select(".platform-icon.bsd").size();
				Integer has_mac = ad.select(".platform-icon.mac").size();
				 
				Elements pi = addtionalInformation.select(".project-info");
				String categories = pi.get(2).select(".content").text();
				 
				int offset = 0;
				String awards = "";
				if (pi.get(3).select("header").text().equals("Awards")) {
					// project is award winning, which introduces a new field
					// Awards
					// have to shift the rest of the field by 1 with the
					// variable offset
					offset = 1;

					// iterate through all the awards and append it to a string
					Elements a = pi.get(3).select("a");
					for (int i = 0; i < a.size(); ++i) {
						if (awards != "")
							awards += "\n";
						awards += a.get(0).attr("title");
					}
				}

				String registered = pi.get(3 + offset).select(".content")
						.text();
				String license = pi.size() > 4 ? pi.get(4 + offset)
						.select(".content").text() : "";
				String languages = pi.size() > 5 ? pi.get(5 + offset)
						.select(".content").text() : "";
				String audience = pi.size() > 6 ? pi.get(6 + offset)
						.select(".content").text() : "";
				String ui = pi.size() > 7 ? pi.get(7 + offset)
						.select(".content").text() : "";
				String pl = pi.size() > 8 ? pi.get(8 + offset)
						.select(".content").text() : "";

				DateParser dp = new DateParser();
				if (dp.parse(updated)) {
					updated = dp.getDateString();
				}*/
				
				try {
					mDataStorageInstance.insertProjectPageDetails(pname, mDate,
							likes, dislikes, updated, has_window, has_mac,
							has_linux, has_bsd, categories, registeredDateStr,
							license, natlanguageStr, audienceStr, uiStr,
							programLanguageStr, awardStr);
					System.out
							.println("    Inserting project page details into MySQL database\n");
				} catch (Exception e) {
					System.out
							.println("    Error inserting into MySQL database!\n");
					e.printStackTrace();
				}
			} catch (Exception e) {
				System.out.println("    Error parsing HTML");
			}
		}
	}

}
